{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Advanced RNN - 2\n",
    "- Objective: try various types of NN architectures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset\n",
    "- CIFAR-10 dataset\n",
    "- source: https://www.cs.toronto.edu/~kriz/cifar.html\n",
    "<img src=\"https://image.slidesharecdn.com/pycon2015-150913033231-lva1-app6892/95/pycon-2015-48-638.jpg?cb=1442115225\" style=\"width: 500px\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from keras.datasets import cifar10\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 32, 32, 3)\n",
      "(10000, 32, 32, 3)\n",
      "(50000, 10)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. CNN-RNN\n",
    "- Perform convolution & pooling operation first, and then perform recurrent operation\n",
    "- Similar to the structure used in image captioning\n",
    "\n",
    "<img src=\"https://cdn-images-1.medium.com/max/1600/1*vzFwXFJOrg6WRGNsYYT6qg.png\" style=\"width: 600px\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, LSTM, Input, Activation, Reshape, concatenate\n",
    "from keras import optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Conv2D(input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3]), filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size = (2,2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 16, 16, 50)\n"
     ]
    }
   ],
   "source": [
    "print(model.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Reshape(target_shape = (16*16, 50)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(LSTM(50, return_sequences = False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dense(10))\n",
    "model.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "adam = optimizers.Adam(lr = 0.001)\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_8 (Conv2D)            (None, 32, 32, 50)        1400      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 32, 32, 50)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2 (None, 16, 16, 50)        0         \n",
      "_________________________________________________________________\n",
      "reshape_6 (Reshape)          (None, 256, 50)           0         \n",
      "_________________________________________________________________\n",
      "lstm_13 (LSTM)               (None, 50)                20200     \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 10)                510       \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 22,110\n",
      "Trainable params: 22,110\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3h 54min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "history = model.fit(X_train, y_train, epochs = 100, batch_size = 100, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 20s    \n"
     ]
    }
   ],
   "source": [
    "results = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy:  0.5927\n"
     ]
    }
   ],
   "source": [
    "print('Test Accuracy: ', results[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. CNN-RNN-2\n",
    "- Perform convolution-pooling operations and recurrent operations independently, and sum their results up\n",
    "- Similar to the structure used in visual question answering\n",
    "\n",
    "<img src=\"https://camo.githubusercontent.com/828817c970da406d2d83dc9a5c03fb120231e2a2/687474703a2f2f692e696d6775722e636f6d2f56627149525a7a2e706e67\" style=\"width: 800px\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_layer = Input(shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3]))\n",
    "conv_layer = Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same')(input_layer)\n",
    "activation_layer = Activation('relu')(conv_layer)\n",
    "pooling_layer = MaxPooling2D(pool_size = (2,2), padding = 'same')(activation_layer)\n",
    "flatten = Flatten()(pooling_layer)\n",
    "dense_layer_1 = Dense(100)(flatten)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reshape = Reshape(target_shape = (X_train.shape[1]*X_train.shape[2], X_train.shape[3]))(input_layer)\n",
    "lstm_layer = LSTM(50, return_sequences = False)(reshape)\n",
    "dense_layer_2 = Dense(100)(lstm_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merged_layer = concatenate([dense_layer_1, dense_layer_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_layer = Dense(10, activation = 'softmax')(merged_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Model(inputs = input_layer, outputs = output_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "adam = optimizers.Adam(lr = 0.001)\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_4 (InputLayer)             (None, 32, 32, 3)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)                (None, 32, 32, 50)    1400        input_4[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "activation_8 (Activation)        (None, 32, 32, 50)    0           conv2d_6[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)   (None, 16, 16, 50)    0           activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "reshape_4 (Reshape)              (None, 1024, 3)       0           input_4[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)              (None, 12800)         0           max_pooling2d_5[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_4 (LSTM)                    (None, 50)            10800       reshape_4[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_6 (Dense)                  (None, 100)           1280100     flatten_2[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_7 (Dense)                  (None, 100)           5100        lstm_4[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)      (None, 200)           0           dense_6[0][0]                    \n",
      "                                                                   dense_7[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_8 (Dense)                  (None, 10)            2010        concatenate_2[0][0]              \n",
      "====================================================================================================\n",
      "Total params: 1,299,410\n",
      "Trainable params: 1,299,410\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "history = model.fit(X_train, y_train, epochs = 10, batch_size = 100, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 74s    \n"
     ]
    }
   ],
   "source": [
    "results = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[14.50628567199707, 0.10000000000000001]\n"
     ]
    }
   ],
   "source": [
    "print('Test Accuracy: ', results[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
